import os
os.getcwd()

'/Users/adityakaul'

Perform the ulimit -a and ulimit -Sn before opening the notebook¶

Before we read the 801 PNG files we need to increase the allowable limit by mac¶

Check the current number of files allowed by using ulimit -a¶

Then use ulimit -Sn 1000 to increase the number of allowed files to 1000¶

#Ideally create different folders for the mons so that you can call only the ones needed for analysis

from PIL import Image
import glob
image_list = []

for filename in glob.glob('/Users/adityakaul/pokemon_data/POKEMON/*.png'): #assuming png
    im=Image.open(filename)
    image_list.append(im)

len(image_list)
#Show the image at the lcoation 103
#image_list[103]

801

#Get the pokedex number of the pokemon since they are randomly present in the list
names = glob.glob('/Users/adityakaul/pokemon_data/POKEMON/*.png')
names[0]

'/Users/adityakaul/pokemon_data/POKEMON/0298.png'

pokedex_no = []
for val in names:
    pokedex_no.append(val[40:43].lstrip("0"))
len(pokedex_no)

801

#get the pokedex number at each location in the list
print(pokedex_no[1])
image_list[1]

267

import pandas as pd
from IPython.display import display

#poke_info.head()

#Get the different features about pokemon from the csv file
poke_info = pd.read_csv("/Users/adityakaul/pokemon_data/pokemon.csv")
pd.options.display.max_columns = None
#display(poke_info)

#display the top 5 rows of data from the dataframe
poke_info.head(5)

non_legendary = poke_info[poke_info['is_legendary'] == 0]
legendary = poke_info[poke_info['is_legendary'] == 1]

legendary_type = legendary.type1.unique().tolist()
non_legendary_type = non_legendary.type1.unique().tolist()

def pokemon_subset(pokemon_type, data):
    df = data
    pokemon_sub = df[(df['type1'] == pokemon_type) | (df['type2'] == pokemon_type)].drop_duplicates()
    pokemon_type_sub = pokemon_sub[['name','pokedex_number','generation','sp_attack','sp_defense','speed','hp','defense','attack']]
    #print(tabulate(pokemon_type_sub, headers='keys', tablefmt='psql'))
    return(pokemon_type_sub)

def strongest_pokemon(pokemon_type, data):
    df = pokemon_subset(pokemon_type, data)
    df['total_stats'] = df['sp_attack'] + df['sp_defense'] + df['speed'] + df['hp'] + df['defense'] + df['attack']
    df['type'] = pokemon_type
    #strongest = df[df['total_stats'] == df['total_stats'].max()]
    strongest = df.loc[df['total_stats'].idxmax()].to_frame().transpose()
    strongest = strongest[['name','type','generation','sp_attack','sp_defense','speed','hp','defense','attack','pokedex_number']]
    print(tabulate(strongest[['name','type','generation','sp_attack','sp_defense','speed','hp','defense','attack']], headers='keys', tablefmt='psql'))
    display(image_list[pokedex_no.index(str(int(strongest.pokedex_number)))])
    #return(strongest)

for val in legendary_type:
    df = strongest_pokemon(val,legendary)

+-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name   | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 645 | Kyurem | ice    |            5 |         170 |          100 |      95 |  125 |        90 |      120 |
+-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+--------+----------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name   | type     |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+--------+----------+--------------+-------------+--------------+---------+------+-----------+----------|
| 643 | Zekrom | electric |            5 |         120 |          100 |      90 |  100 |       120 |      150 |
+-----+--------+----------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name   | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 249 | Ho-Oh  | fire   |            2 |         110 |          154 |      90 |  106 |        90 |      130 |
+-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+--------+---------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name   | type    |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+--------+---------+--------------+-------------+--------------+---------+------+-----------+----------|
| 149 | Mewtwo | psychic |            1 |         194 |          120 |     140 |  106 |        70 |      150 |
+-----+--------+---------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name   | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 381 | Kyogre | water  |            3 |         180 |          160 |      90 |  100 |        90 |      150 |
+-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+---------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name    | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+---------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 718 | Diancie | rock   |            6 |         160 |          110 |     110 |   50 |       110 |      160 |
+-----+---------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name   | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 482 | Dialga | steel  |            4 |         150 |          100 |      90 |  100 |       120 |      120 |
+-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name     | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 383 | Rayquaza | dragon |            3 |         180 |          100 |     115 |  105 |       100 |      180 |
+-----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+---------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name    | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+---------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 382 | Groudon | ground |            3 |         150 |           90 |      90 |  100 |       160 |      180 |
+-----+---------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name   | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 492 | Arceus | normal |            4 |         120 |          120 |     120 |  120 |       120 |      120 |
+-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

poke_info.columns

Index(['abilities', 'against_bug', 'against_dark', 'against_dragon',
       'against_electric', 'against_fairy', 'against_fight', 'against_fire',
       'against_flying', 'against_ghost', 'against_grass', 'against_ground',
       'against_ice', 'against_normal', 'against_poison', 'against_psychic',
       'against_rock', 'against_steel', 'against_water', 'attack',
       'base_egg_steps', 'base_happiness', 'base_total', 'capture_rate',
       'classfication', 'defense', 'experience_growth', 'height_m', 'hp',
       'japanese_name', 'name', 'percentage_male', 'pokedex_number',
       'sp_attack', 'sp_defense', 'speed', 'type1', 'type2', 'weight_kg',
       'generation', 'is_legendary'],
      dtype='object')

for val in non_legendary_type:
    df = strongest_pokemon(val,non_legendary)

+-----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name     | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 253 | Sceptile | grass  |            3 |         145 |           85 |     145 |   70 |        75 |      110 |
+-----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+----+-----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|    | name      | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|----+-----------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
|  5 | Charizard | fire   |            1 |         159 |          115 |     100 |   78 |        78 |      104 |
+----+-----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name     | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 129 | Gyarados | water  |            1 |          70 |          130 |      81 |   95 |       109 |      155 |
+-----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name   | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 126 | Pinsir | bug    |            1 |          65 |           90 |     105 |   65 |       120 |      155 |
+-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+---------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name    | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+---------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 288 | Slaking | normal |            3 |          95 |           65 |     100 |  150 |       100 |      160 |
+-----+---------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|    | name     | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
|  2 | Venusaur | poison |            1 |         122 |          120 |      80 |   80 |       123 |      100 |
+----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+----------+----------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name     | type     |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+----------+----------+--------------+-------------+--------------+---------+------+-----------+----------|
| 180 | Ampharos | electric |            2 |         165 |          110 |      45 |   90 |       105 |       95 |
+-----+----------+----------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name     | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 444 | Garchomp | ground |            4 |         120 |           95 |      92 |  108 |       115 |      170 |
+-----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+-----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name      | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+-----------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 281 | Gardevoir | fairy  |            3 |         165 |          135 |     100 |   68 |        65 |       85 |
+-----+-----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+----------+----------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name     | type     |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+----------+----------+--------------+-------------+--------------+---------+------+-----------+----------|
| 256 | Blaziken | fighting |            3 |         130 |           80 |     100 |   80 |        80 |      160 |
+-----+----------+----------+--------------+-------------+--------------+---------+------+-----------+----------+

def generation_mon(gen):
    gen_sub = poke_info[(poke_info['generation'] == gen) & (poke_info['is_legendary'] != 1)]
    for val in non_legendary_type:
        df = strongest_pokemon(val,gen_sub)
    return(df)

gen_list = poke_info.generation.unique().tolist()

for val in gen_list:
    generation_mon(val)

+----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|    | name     | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
|  2 | Venusaur | grass  |            1 |         122 |          120 |      80 |   80 |       123 |      100 |
+----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+----+-----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|    | name      | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|----+-----------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
|  5 | Charizard | fire   |            1 |         159 |          115 |     100 |   78 |        78 |      104 |
+----+-----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name     | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 129 | Gyarados | water  |            1 |          70 |          130 |      81 |   95 |       109 |      155 |
+-----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name   | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 126 | Pinsir | bug    |            1 |          65 |           90 |     105 |   65 |       120 |      155 |
+-----+--------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+------------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name       | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+------------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 114 | Kangaskhan | normal |            1 |          60 |          100 |     100 |  105 |       100 |      125 |
+-----+------------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|    | name     | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
|  2 | Venusaur | poison |            1 |         122 |          120 |      80 |   80 |       123 |      100 |
+----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+-----+---------+----------+--------------+-------------+--------------+---------+------+-----------+----------+
|     | name    | type     |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|-----+---------+----------+--------------+-------------+--------------+---------+------+-----------+----------|
| 134 | Jolteon | electric |            1 |         110 |           95 |     130 |   65 |        60 |       65 |
+-----+---------+----------+--------------+-------------+--------------+---------+------+-----------+----------+

+----+-----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|    | name      | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|----+-----------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 30 | Nidoqueen | ground |            1 |          75 |           85 |      76 |   90 |        87 |       92 |
+----+-----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+
|    | name     | type   |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------|
| 35 | Clefable | fairy  |            1 |          95 |           90 |      60 |   95 |        73 |       70 |
+----+----------+--------+--------------+-------------+--------------+---------+------+-----------+----------+

+----+-----------+----------+--------------+-------------+--------------+---------+------+-----------+----------+
|    | name      | type     |   generation |   sp_attack |   sp_defense |   speed |   hp |   defense |   attack |
|----+-----------+----------+--------------+-------------+--------------+---------+------+-----------+----------|
| 61 | Poliwrath | fighting |            1 |          70 |           90 |      70 |   90 |        95 |       95 |
+----+-----------+----------+--------------+-------------+--------------+---------+------+-----------+----------+

non_legendary.head()
non_legendary_mod = non_legendary

import warnings
warnings.filterwarnings('ignore')

non_legendary_mod['total_weakness'] = non_legendary_mod['against_bug'] + non_legendary_mod['against_dark'] + \
non_legendary_mod['against_dragon'] + non_legendary_mod['against_electric'] + non_legendary_mod['against_fairy'] + \
non_legendary_mod['against_fight'] + non_legendary_mod['against_fire'] + non_legendary_mod['against_flying'] + \
non_legendary_mod['against_ghost'] + non_legendary_mod['against_grass'] + non_legendary_mod['against_ground'] + \
non_legendary_mod['against_ice'] + non_legendary_mod['against_normal'] + non_legendary_mod['against_poison'] + \
non_legendary_mod['against_psychic'] + non_legendary_mod['against_rock'] + non_legendary_mod['against_steel'] + \
non_legendary_mod['against_water']

weakest_mon = non_legendary_mod.loc[non_legendary_mod['total_weakness'].idxmax()].to_frame().transpose()
weakest_mon = weakest_mon[['name','pokedex_number','generation','classfication','type1','type2','total_weakness']]
print(tabulate(weakest_mon, headers='keys', tablefmt='psql'))
display(image_list[pokedex_no.index(str(int(weakest_mon.pokedex_number)))])

+-----+--------+------------------+--------------+-----------------+---------+---------+------------------+
|     | name   |   pokedex_number |   generation | classfication   | type1   | type2   |   total_weakness |
|-----+--------+------------------+--------------+-----------------+---------+---------+------------------|
| 697 | Amaura |              698 |            6 | Tundra Pokémon  | rock    | ice     |               26 |
+-----+--------+------------------+--------------+-----------------+---------+---------+------------------+

strongest_mon = non_legendary_mod.loc[non_legendary_mod['total_weakness'].idxmin()].to_frame().transpose()
strongest_mon = strongest_mon[['name','pokedex_number','generation','classfication','type1','type2','total_weakness']]
print(tabulate(strongest_mon, headers='keys', tablefmt='psql'))
display(image_list[pokedex_no.index(str(int(strongest_mon.pokedex_number)))])

+-----+--------+------------------+--------------+------------------+---------+---------+------------------+
|     | name   |   pokedex_number |   generation | classfication    | type1   | type2   |   total_weakness |
|-----+--------+------------------+--------------+------------------+---------+---------+------------------|
| 302 | Mawile |              303 |            3 | Deceiver Pokémon | steel   | fairy   |            13.25 |
+-----+--------+------------------+--------------+------------------+---------+---------+------------------+

heaviest = non_legendary_mod.loc[non_legendary_mod['weight_kg'].idxmax()].to_frame().transpose()
heaviest = heaviest[['name','pokedex_number','generation','classfication','type1','type2','weight_kg']]
print(tabulate(heaviest, headers='keys', tablefmt='psql'))
display(image_list[pokedex_no.index(str(int(heaviest.pokedex_number)))])

+-----+----------+------------------+--------------+---------------------+---------+---------+-------------+
|     | name     |   pokedex_number |   generation | classfication       | type1   |   type2 |   weight_kg |
|-----+----------+------------------+--------------+---------------------+---------+---------+-------------|
| 749 | Mudsdale |              750 |            7 | Draft Horse Pokémon | ground  |     nan |         920 |
+-----+----------+------------------+--------------+---------------------+---------+---------+-------------+

lightest = non_legendary_mod.loc[non_legendary_mod['weight_kg'].idxmin()].to_frame().transpose()
lightest = lightest[['name','pokedex_number','generation','classfication','type1','type2','weight_kg']]
print(tabulate(lightest, headers='keys', tablefmt='psql'))
display(image_list[pokedex_no.index(str(int(lightest.pokedex_number)))])

+----+--------+------------------+--------------+-----------------+---------+---------+-------------+
|    | name   |   pokedex_number |   generation | classfication   | type1   | type2   |   weight_kg |
|----+--------+------------------+--------------+-----------------+---------+---------+-------------|
| 91 | Gastly |               92 |            1 | Gas Pokémon     | ghost   | poison  |         0.1 |
+----+--------+------------------+--------------+-----------------+---------+---------+-------------+

tallest = non_legendary_mod.loc[non_legendary_mod['height_m'].idxmax()].to_frame().transpose()
tallest = tallest[['name','pokedex_number','generation','classfication','type1','type2','height_m']]
print(tabulate(tallest, headers='keys', tablefmt='psql'))
display(image_list[pokedex_no.index(str(int(tallest.pokedex_number)))])

+-----+---------+------------------+--------------+---------------------+---------+---------+------------+
|     | name    |   pokedex_number |   generation | classfication       | type1   |   type2 |   height_m |
|-----+---------+------------------+--------------+---------------------+---------+---------+------------|
| 320 | Wailord |              321 |            3 | Float Whale Pokémon | water   |     nan |       14.5 |
+-----+---------+------------------+--------------+---------------------+---------+---------+------------+

shortest = non_legendary_mod.loc[non_legendary_mod['height_m'].idxmin()].to_frame().transpose()
shortest = shortest[['name','pokedex_number','generation','classfication','type1','type2','height_m']]
print(tabulate(shortest, headers='keys', tablefmt='psql'))
display(image_list[pokedex_no.index(str(int(shortest.pokedex_number)))])

+-----+--------+------------------+--------------+-------------------+---------+----------+------------+
|     | name   |   pokedex_number |   generation | classfication     | type1   | type2    |   height_m |
|-----+--------+------------------+--------------+-------------------+---------+----------+------------|
| 594 | Joltik |              595 |            5 | Attaching Pokémon | bug     | electric |        0.1 |
+-----+--------+------------------+--------------+-------------------+---------+----------+------------+

non_legendary.head()

non_legendary_mon1 = non_legendary[['type1','name']]
non_legendary_mon2 = non_legendary[['type2','name']]
type1 = non_legendary_mon1.groupby(['type1']).count().reset_index()
type2 = non_legendary_mon2.groupby(['type2']).count().reset_index()

type_mons = type1.merge(type2, left_on='type1', right_on='type2', how='outer')
type_mons['no_of_mons'] = type_mons['name_x'] + type_mons['name_y']
type_mons = type_mons[['type1','no_of_mons']]

type_mons.transpose()

import matplotlib.pyplot as plt; plt.rcdefaults()
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

type_mons.plot.barh(x='type1', y='no_of_mons', rot=1, color = 'steelblue')
plt.show()

legendary_mon1 = legendary[['type1','name']]
legendary_mon2 = legendary[['type2','name']]
type1 = legendary_mon1.groupby(['type1']).count().reset_index()
type2 = legendary_mon2.groupby(['type2']).count().reset_index()

type_mons = type1.merge(type2, left_on='type1', right_on='type2', how='outer')
type_mons['no_of_mons'] = type_mons['name_x'] + type_mons['name_y']
type_mons = type_mons[['type1','no_of_mons']]

type_mons = type_mons[pd.notnull(type_mons['type1'])]
type_mons.transpose().fillna(0)

type_mons.plot.barh(x='type1', y='no_of_mons', rot=1, color = 'steelblue')
plt.show()

#perform clustering to see if all pokemon are classified based on tyoe properly
#perform some predictions to see if a regulr pokemon can be legendary based on features?
#poke_info['abilities'][0].strip('\"')
#poke_info['abilities'].unique()
#poke_info['capture_rate'].unique()
poke_info.capture_rate.iloc[[773]] = '255'
poke_info['capture_rate'].unique()

array(['45', '255', '120', '127', '90', '190', '75', '235', '150', '25',
       '170', '50', '200', '100', '180', '60', '225', '30', '35', '3',
       '65', '70', '125', '205', '155', '145', '130', '140', '15', '220',
       '160', '80', '55'], dtype=object)

poke_info_ml = poke_info[['against_bug', 'against_dark', 'against_dragon',
       'against_electric', 'against_fairy', 'against_fight', 'against_fire',
       'against_flying', 'against_ghost', 'against_grass', 'against_ground',
       'against_ice', 'against_normal', 'against_poison', 'against_psychic',
       'against_rock', 'against_steel', 'against_water', 'attack',
       'base_egg_steps', 'base_happiness', 'base_total', 'capture_rate',
       'defense', 'experience_growth', 'height_m', 'hp',
       'sp_attack', 'sp_defense', 'speed', 'weight_kg',
       'is_legendary']]

poke_info_ml.head()
poke_info_ml_clean = poke_info_ml.dropna()
len(poke_info_ml_clean)

781

data_final_vars=poke_info_ml_clean.columns.values.tolist()
Y=['is_legendary']
X=[i for i in data_final_vars if i not in Y]

from sklearn.feature_selection import RFE
from sklearn.linear_model import LogisticRegression

logreg = LogisticRegression()

rfe = RFE(logreg, 18)
rfe = rfe.fit(poke_info_ml_clean[X], poke_info_ml_clean[Y] )
print(rfe.support_)
print(rfe.ranking_)

[ True  True  True False False  True False False False  True False  True
 False False False False False False  True  True  True  True  True  True
  True False  True  True  True  True  True]
[ 1  1  1  8  3  1  7  4 14  1 10  1  9 13  6  2 11  5  1  1  1  1  1  1
  1 12  1  1  1  1  1]

poke_info_ml_useful = poke_info[['against_bug', 'against_dark', 'against_dragon',
       'against_fight',
       'against_grass',
       'against_ice',
       'attack',
       'base_egg_steps', 'base_happiness', 'base_total', 'capture_rate',
       'defense', 'experience_growth', 'hp',
       'sp_attack', 'sp_defense', 'speed', 'weight_kg',
       'is_legendary']]
poke_info_ml_useful.head()

#poke_info_ml_useful.head()
poke_info_ml_useful.dtypes

against_bug           float64
against_dark          float64
against_dragon        float64
against_fight         float64
against_grass         float64
against_ice           float64
attack                  int64
base_egg_steps          int64
base_happiness          int64
base_total              int64
capture_rate            int64
defense                 int64
experience_growth       int64
hp                      int64
sp_attack               int64
sp_defense              int64
speed                   int64
weight_kg             float64
is_legendary         category
dtype: object

poke_info_ml_useful[['against_bug', 'against_dark', 'against_dragon',
       'against_fight',
       'against_grass',
       'against_ice',
       'attack',
       'base_egg_steps', 'base_happiness', 'base_total', 'capture_rate',
       'defense', 'experience_growth', 'hp',
       'sp_attack', 'sp_defense', 'speed', 'weight_kg']] = poke_info_ml_useful[['against_bug', 'against_dark', 'against_dragon',
       'against_fight',
       'against_grass',
       'against_ice',
       'attack',
       'base_egg_steps', 'base_happiness', 'base_total', 'capture_rate',
       'defense', 'experience_growth', 'hp',
       'sp_attack', 'sp_defense', 'speed', 'weight_kg']].apply(pd.to_numeric)

poke_info_ml_useful[['is_legendary']] = poke_info_ml_useful[['is_legendary']].astype('category')

poke_info_ml_useful_vars = poke_info_ml_useful.columns.values.tolist()
Y=['is_legendary']
X=[i for i in poke_info_ml_useful_vars if i not in Y]

poke_info_ml_useful = poke_info_ml_useful.dropna()
len(poke_info_ml_useful)

781

import statsmodels.api as sm
logit_model=sm.Logit(poke_info_ml_useful[Y],poke_info_ml_useful[X])
result=logit_model.fit()
print(result.summary())

Optimization terminated successfully.
         Current function value: 0.041552
         Iterations 33
                           Logit Regression Results                           
==============================================================================
Dep. Variable:           is_legendary   No. Observations:                  781
Model:                          Logit   Df Residuals:                      764
Method:                           MLE   Df Model:                           16
Date:                Mon, 10 Sep 2018   Pseudo R-squ.:                  0.8609
Time:                        00:44:49   Log-Likelihood:                -32.452
converged:                       True   LL-Null:                       -233.28
                                        LLR p-value:                 1.631e-75
=====================================================================================
                        coef    std err          z      P>|z|      [0.025      0.975]
-------------------------------------------------------------------------------------
against_bug           0.5552      1.012      0.548      0.583      -1.429       2.539
against_dark         -2.5488      0.851     -2.994      0.003      -4.217      -0.880
against_dragon       -2.6153      0.949     -2.755      0.006      -4.476      -0.755
against_fight        -2.4059      0.990     -2.431      0.015      -4.346      -0.466
against_grass        -0.1573      0.669     -0.235      0.814      -1.468       1.154
against_ice          -1.2348      0.786     -1.570      0.116      -2.776       0.307
attack                0.0031        nan        nan        nan         nan         nan
base_egg_steps        0.0004   7.27e-05      5.918      0.000       0.000       0.001
base_happiness        0.0003      0.016      0.021      0.983      -0.030       0.031
base_total            0.0028        nan        nan        nan         nan         nan
capture_rate         -0.0362      0.009     -4.155      0.000      -0.053      -0.019
defense               0.0154        nan        nan        nan         nan         nan
experience_growth  4.183e-07    2.2e-06      0.190      0.849    -3.9e-06    4.73e-06
hp                   -0.0094        nan        nan        nan         nan         nan
sp_attack             0.0090        nan        nan        nan         nan         nan
sp_defense           -0.0206        nan        nan        nan         nan         nan
speed                 0.0050        nan        nan        nan         nan         nan
weight_kg             0.0031      0.004      0.756      0.450      -0.005       0.011
=====================================================================================

Possibly complete quasi-separation: A fraction 0.34 of observations can be
perfectly predicted. This might indicate that there is complete
quasi-separation. In this case some parameters will not be identified.

from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(poke_info_ml_useful[X], np.ravel(poke_info_ml_useful[Y]), test_size=0.3, random_state=0)
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
logreg = LogisticRegression()
logreg.fit(X_train, y_train)

LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,
          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,
          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,
          verbose=0, warm_start=False)

y_pred = logreg.predict(X_test)
print('Accuracy of logistic regression classifier on test set: {:.2f}'.format(logreg.score(X_test, y_test)))

Accuracy of logistic regression classifier on test set: 0.98

len(X_train)
len(X_test)

235

from sklearn import model_selection
from sklearn.model_selection import cross_val_score

kfold = model_selection.KFold(n_splits=10, random_state=7)
modelCV = LogisticRegression()
scoring = 'accuracy'
results = model_selection.cross_val_score(modelCV, X_train, y_train, cv=kfold, scoring=scoring)
print("10-fold cross validation average accuracy: %.3f" % (results.mean()))

10-fold cross validation average accuracy: 0.991

from sklearn.metrics import confusion_matrix
confusion_matrix = confusion_matrix(y_test, y_pred)
print(confusion_matrix)

[[208   1]
 [  3  23]]

tp = confusion_matrix[0][0]
fp = confusion_matrix[0][1]
fn = confusion_matrix[1][0]
tn = confusion_matrix[1][1]

accuracy = (tp + tn)/(tp + tn + fp + fn)
print("Accuracy: " + str(round(accuracy*100,2)) + "%")

precision = (tp)/(tp + fp)
print("Precision: " + str(round(precision*100,2)) + "%")

recall = (tp)/(tp + fn)
print("Recall: " + str(round(recall*100,2)) + "%")

f1_score = (2*precision*recall)/(precision + recall)
print("F1 Score: " + str(round(f1_score*100,2)) + "%")

Accuracy: 98.3%
Precision: 99.52%
Recall: 98.58%
F1 Score: 99.05%

from sklearn.metrics import classification_report
print(classification_report(y_test, y_pred))

             precision    recall  f1-score   support

          0       0.99      1.00      0.99       209
          1       0.96      0.88      0.92        26

avg / total       0.98      0.98      0.98       235

from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve
logit_roc_auc = roc_auc_score(y_test, logreg.predict(X_test))
fpr, tpr, thresholds = roc_curve(y_test, logreg.predict_proba(X_test)[:,1])
plt.figure()
plt.plot(fpr, tpr, label='Logistic Regression (area = %0.2f)' % logit_roc_auc)
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic')
plt.legend(loc="lower right")
plt.savefig('Log_ROC')
plt.show()

X_test
pd.DataFrame(y_test)
pd.DataFrame(y_pred)

X_test['y_test'] = Series(y_test, index=X_test.index)
X_test['y_pred'] = Series(y_pred, index=X_test.index)
X_test.head()

#find which ones were wrongly classified

wrongly_classified = X_test[(X_test['y_pred'] == 0) & (X_test['y_test'] == 1)].index.values

for index_val in wrongly_classified:
    print(tabulate(poke_info[['name','pokedex_number','type1','type2','generation']].iloc[index_val].to_frame().transpose(), headers='keys', tablefmt='psql'))

+-----+-----------+------------------+---------+---------+--------------+
|     | name      |   pokedex_number | type1   | type2   |   generation |
|-----+-----------+------------------+---------+---------+--------------|
| 785 | Tapu Lele |              786 | psychic | fairy   |            7 |
+-----+-----------+------------------+---------+---------+--------------+
+-----+---------+------------------+---------+---------+--------------+
|     | name    |   pokedex_number | type1   | type2   |   generation |
|-----+---------+------------------+---------+---------+--------------|
| 718 | Diancie |              719 | rock    | fairy   |            6 |
+-----+---------+------------------+---------+---------+--------------+
+-----+---------+------------------+---------+---------+--------------+
|     | name    |   pokedex_number | type1   |   type2 |   generation |
|-----+---------+------------------+---------+---------+--------------|
| 489 | Manaphy |              490 | water   |     nan |            4 |
+-----+---------+------------------+---------+---------+--------------+

wrongly_classified_non_leg = X_test[(X_test['y_pred'] == 1) & (X_test['y_test'] == 0)].index.values
for index_val in wrongly_classified_non_leg:
    print(tabulate(poke_info[['name','pokedex_number','type1','type2','generation']].iloc[index_val].to_frame().transpose(), headers='keys', tablefmt='psql'))

+-----+----------+------------------+---------+---------+--------------+
|     | name     |   pokedex_number | type1   |   type2 |   generation |
|-----+----------+------------------+---------+---------+--------------|
| 772 | Silvally |              773 | normal  |     nan |            7 |
+-----+----------+------------------+---------+---------+--------------+

	abilities	against_bug	against_dark	against_dragon	against_electric	against_fairy	against_fight	against_fire	against_flying	against_ghost	against_grass	against_ground	against_ice	against_normal	against_poison	against_psychic	against_rock	against_steel	against_water	attack	base_egg_steps	base_happiness	base_total	capture_rate	classfication	defense	experience_growth	height_m	hp	japanese_name	name	percentage_male	pokedex_number	sp_attack	sp_defense	speed	type1	type2	weight_kg	generation
0	['Overgrow', 'Chlorophyll']	1.0	1.0	1.0	0.5	0.5	0.5	2.0	2.0	1.0	0.25	1.0	2.0	1.0	1.0	2.0	1.0	1.0	0.5	49	5120	70	318	45	Seed Pokémon	49	1059860	0.7	45	Fushigidaneフシギダネ	Bulbasaur	88.1	1	65	65	45	grass	poison	6.9	1
1	['Overgrow', 'Chlorophyll']	1.0	1.0	1.0	0.5	0.5	0.5	2.0	2.0	1.0	0.25	1.0	2.0	1.0	1.0	2.0	1.0	1.0	0.5	62	5120	70	405	45	Seed Pokémon	63	1059860	1.0	60	Fushigisouフシギソウ	Ivysaur	88.1	2	80	80	60	grass	poison	13.0	1
2	['Overgrow', 'Chlorophyll']	1.0	1.0	1.0	0.5	0.5	0.5	2.0	2.0	1.0	0.25	1.0	2.0	1.0	1.0	2.0	1.0	1.0	0.5	100	5120	70	625	45	Seed Pokémon	123	1059860	2.0	80	Fushigibanaフシギバナ	Venusaur	88.1	3	122	120	80	grass	poison	100.0	1
3	['Blaze', 'Solar Power']	0.5	1.0	1.0	1.0	0.5	1.0	0.5	1.0	1.0	0.50	2.0	0.5	1.0	1.0	1.0	2.0	0.5	2.0	52	5120	70	309	45	Lizard Pokémon	43	1059860	0.6	39	Hitokageヒトカゲ	Charmander	88.1	4	60	50	65	fire	NaN	8.5	1
4	['Blaze', 'Solar Power']	0.5	1.0	1.0	1.0	0.5	1.0	0.5	1.0	1.0	0.50	2.0	0.5	1.0	1.0	1.0	2.0	0.5	2.0	64	5120	70	405	45	Flame Pokémon	58	1059860	1.1	58	Lizardoリザード	Charmeleon	88.1	5	80	65	80	fire	NaN	19.0	1

	abilities	against_bug	against_dark	against_dragon	against_electric	against_fairy	against_fight	against_fire	against_flying	against_ghost	against_grass	against_ground	against_ice	against_normal	against_poison	against_psychic	against_rock	against_steel	against_water	attack	base_egg_steps	base_happiness	base_total	capture_rate	classfication	defense	experience_growth	height_m	hp	japanese_name	name	percentage_male	pokedex_number	sp_attack	sp_defense	speed	type1	type2	weight_kg	generation	total_weakness
0	['Overgrow', 'Chlorophyll']	1.0	1.0	1.0	0.5	0.5	0.5	2.0	2.0	1.0	0.25	1.0	2.0	1.0	1.0	2.0	1.0	1.0	0.5	49.0	5120.0	70.0	318.0	45	Seed Pokémon	49.0	1059860.0	0.7	45.0	Fushigidaneフシギダネ	Bulbasaur	88.1	1.0	65.0	65.0	45.0	grass	poison	6.9	1.0	19.25
1	['Overgrow', 'Chlorophyll']	1.0	1.0	1.0	0.5	0.5	0.5	2.0	2.0	1.0	0.25	1.0	2.0	1.0	1.0	2.0	1.0	1.0	0.5	62.0	5120.0	70.0	405.0	45	Seed Pokémon	63.0	1059860.0	1.0	60.0	Fushigisouフシギソウ	Ivysaur	88.1	2.0	80.0	80.0	60.0	grass	poison	13.0	1.0	19.25
2	['Overgrow', 'Chlorophyll']	1.0	1.0	1.0	0.5	0.5	0.5	2.0	2.0	1.0	0.25	1.0	2.0	1.0	1.0	2.0	1.0	1.0	0.5	100.0	5120.0	70.0	625.0	45	Seed Pokémon	123.0	1059860.0	2.0	80.0	Fushigibanaフシギバナ	Venusaur	88.1	3.0	122.0	120.0	80.0	grass	poison	100.0	1.0	19.25
3	['Blaze', 'Solar Power']	0.5	1.0	1.0	1.0	0.5	1.0	0.5	1.0	1.0	0.50	2.0	0.5	1.0	1.0	1.0	2.0	0.5	2.0	52.0	5120.0	70.0	309.0	45	Lizard Pokémon	43.0	1059860.0	0.6	39.0	Hitokageヒトカゲ	Charmander	88.1	4.0	60.0	50.0	65.0	fire	NaN	8.5	1.0	18.00
4	['Blaze', 'Solar Power']	0.5	1.0	1.0	1.0	0.5	1.0	0.5	1.0	1.0	0.50	2.0	0.5	1.0	1.0	1.0	2.0	0.5	2.0	64.0	5120.0	70.0	405.0	45	Flame Pokémon	58.0	1059860.0	1.1	58.0	Lizardoリザード	Charmeleon	88.1	5.0	80.0	65.0	80.0	fire	NaN	19.0	1.0	18.00

	0	1	2	3	4	5	6	7	8	9	10	11	12	13	14	15	16	17
type1	bug	dark	dragon	electric	fairy	fighting	fire	flying	ghost	grass	ground	ice	normal	poison	psychic	rock	steel	water
no_of_mons	74	47	33	42	40	47	58	87	38	92	63	35	106	65	61	55	36	124

	against_bug	against_dark	against_dragon	against_fight	against_grass	against_ice	attack	base_egg_steps	base_happiness	base_total	capture_rate	defense	experience_growth	hp	sp_attack	sp_defense	speed	weight_kg
400	1.0	1.0	1.0	0.50	0.50	1.0	25	3840	70	194	255	41	1059860	37	25	41	25	2.2
505	1.0	1.0	1.0	2.00	1.00	1.0	60	3840	70	275	255	45	1059860	45	25	45	55	4.1
369	1.0	1.0	1.0	1.00	2.00	0.5	30	5120	70	330	225	55	800000	43	40	65	97	8.7
14	0.5	1.0	1.0	0.25	0.25	1.0	150	3840	70	495	45	40	1000000	65	15	80	145	29.5
163	0.5	1.0	1.0	1.00	0.50	2.0	50	3840	70	452	90	50	1000000	100	86	96	70	40.8